# -*- coding: utf-8 -*-
import sys
import scrapy
import json
import time
import re
import datetime
from lxml import etree
from weibo_scrapy.items import MblogItem, UserItem
from weibo_scrapy.settings import LIMIT_DATE,COOKIES_ENABLED,COOKIES
from weibo_scrapy.login import login
from elasticsearch import Elasticsearch
HEADERS = {
    'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
}

class MwbSpider(scrapy.Spider):
    name = 'swb_2'
    allowed_domains = ['weibo.cn','m.weibo.cn']
    # start_urls = ['http://m.weibo.cn/']

    def start_requests(self):
        words = ['万科A']
        for word in words:
            # 全部
            url = 'https://weibo.cn/search/mblog?hideSearchFrame=&keyword={word}&page={page}'.format(
                word=word,page=1)
            yield scrapy.Request(url=url, cookies=COOKIES,callback=self.parse_first)
    def parse_first(self,response):
        print(response.url)
        mblog_list = response.xpath('//div[@class="c" and starts-with(@id,"M_")]')
        for mblog in mblog_list:
            bid = mblog.xpath('./@id').extract_first().lstrip('M_')
            yield scrapy.Request(url='https://m.weibo.cn/statuses/show?id={}'.format(bid),callback=self.parse_item)
        next_page = response.xpath('//a[text()="下页"]/@href')
        if next_page:
            next_page = 'https://weibo.cn' + next_page.extract_first()
            yield scrapy.Request(url=next_page,callback=self.parse_first)
    def parse_item(self, response):
        print(response.url)
        response_data = json.loads(response.text)
        data = response_data.get('data', '')
        ok = response_data.get('ok', '')
        errno = response_data.get('errno','')
        # if not ok and errno=="100005":
        #     print('403')
        if not ok or not data:
            # print('******************', ok)
            return
        # print(data)
        # return
        if 'statuses/show' in response.url:
            mblog,user = card_deal(data)
            if not mblog['is_top'] and mblog['created_at'] < LIMIT_DATE:
                print('*********************', mblog['created_at'])
                return
            yield MblogItem(mblog)
            # yield UserItem(user)
def card_deal(card):
    mblog = card.get('mblog', card)
    mblog_item = mblog_deal(mblog)
    user = mblog.get('user', '')
    user_item = user_deal(user)
    mblog_item['uid'] = user_item['uid']
    mblog_item['screen_name'] = user_item['screen_name']
    crawl_date = ts2date(time.time())
    mblog_item['crawl_date'] = user_item['crawl_date'] = crawl_date[:10]
    return mblog_item,user_item

def mblog_deal(mblog):
    mid = mblog['mid']
    bid = mblog['bid']
    scheme = 'https://m.weibo.cn/status/' + bid
    comments_count = mblog['comments_count']
    attitudes_count = mblog['attitudes_count']
    reposts_count = mblog['reposts_count']
    created_at, timestamp = trans_date_ts(mblog['created_at'])
    source = mblog.get('source', '')
    # 1原创 2评论 3转发
    weibo_position = mblog.get('weibo_position',0)
    retweeted_status = mblog.get('retweeted_status', '')
    pics = mblog.get('pics', '')
    pic = mblog.get('pic', '')
    pics_count = len(pics) if pics else (len(pic) if pic else 0)
    if retweeted_status:
        root_mid = retweeted_status['id']
        retweeted_user = retweeted_status.get('user', '')
        if retweeted_user:
            root_uid = retweeted_user['id']
        else:
            root_uid = ''
    else:
        root_mid = ''
        root_uid = ''
    tree = etree.HTML(mblog['text'])
    text = ''.join(tree.xpath('.//text() | .//img/@alt'))
    hashtags = re.findall('#(.*?)#', text)
    if hashtags:
        hashtags_count = len(set(hashtags))
        hashtags_content = ','.join(set(hashtags))
    else:
        hashtags_count = 0
        hashtags_content = ''
    is_top = mblog.get('isTop',0)
    item = {
        'mid': mid,
        'scheme': scheme,
        'comments_count': comments_count,
        'attitudes_count': attitudes_count,
        'reposts_count':reposts_count,
        'created_at':created_at,
        'timestamp':timestamp,
        'source':source,
        'weibo_position':weibo_position,
        'pics_count':pics_count,
        'root_mid':root_mid,
        'root_uid':root_uid,
        'text':text,
        'hashtags_count':hashtags_count,
        'hashtags_content':hashtags_content,
        'is_top':is_top
            }
    return item

def user_deal(user):
    item = {}
    item['uid'] = user['id']
    # 用户昵称
    item['screen_name'] = user['screen_name']
    # 用户主页
    item['profile_url'] = user['profile_url']
    # 用户描述
    item['description'] = user['description']
    # 用户关注数
    item['follow_count'] = user['follow_count']
    # 用户粉丝数
    item['followers_count'] = user['followers_count']
    # 发帖数
    item['statuses_count'] = user['statuses_count']
    # 用户性别
    item['gender'] = user['gender']
    # 用户认证信息
    item['verified'] = user['verified']
    item['verified_type'] = user['verified_type']
    item['verified_reason'] = user.get('verified_reason', '')
    return item
def trans_date_ts(date):
    date = date.replace('年','-').replace('月','-').replace('日','')
    # 转成时间戳
    if '分钟前' in date:
        now_ts = time.time()
        minute = int(date.rstrip('分钟前'))
        ts = now_ts - minute * 60
        date = ts2date(ts)
    elif '小时前' in date:
        now_ts = time.time()
        hour = int(date.rstrip('小时前'))
        ts = now_ts - hour * 3600
        date = ts2date(ts)
    elif '昨天' in date:
        now_ts = time.time()
        new_time = date.lstrip('昨天')
        date = get_before_date(1) + new_time + ":00"
    elif '今天' in date:
        now_ts = time.time()
        new_time = date.lstrip('今天')
        date = get_before_date(0) + new_time + ":00"
    elif date.count('-') == 1:
        date = time.strftime("%Y-", time.localtime()) + date+ ":00"
    elif '刚刚' in date:
        date = ts2date(time.time())
    elif '+0800' in date:
        format_time = time.strptime(date, '%a %b %d %H:%M:%S +0800 %Y')
        date = time.strftime('%Y-%m-%d %H:%M:%S', format_time)
    return date[:10], date2ts(date)


def date2ts(date):
    try:
        return int(time.mktime(time.strptime(date, '%Y-%m-%d')))
    except:
        return int(time.mktime(time.strptime(date, '%Y-%m-%d %H:%M:%S')))


def ts2date(ts):
    return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(ts))


def get_before_date(date_cha):
    # date_cha为当天日期的前几天,返回前几天的日期
    # 如当天2018-7-7,date_cha=1,则返回2018-7-6
    today = datetime.date.today()
    oneday = datetime.timedelta(days=date_cha)
    newday = today - oneday
    return str(newday)


def insert(uid,page):
    body = {'uid':uid,'page':page}
    es.index(index='quchong',doc_type='mwb',id=uid,body=body)
def search(uid):
    try:
        r = es.get(index='quchong',doc_type='mwb',id=uid)
        return r['_source']['page']
    except:
        return


# if __name__ == '__main__':
#     print trans_date_ts('刚刚')
